{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Architecture Search with DARTS\n",
    "\n",
    "In this example you will deploy Katib Experiment with Differentiable Architecture Search (DARTS) algorithm using Jupyter Notebook and Katib SDK. Your Kubernetes cluster must have at least one GPU for this example.\n",
    "\n",
    "You can read more about how we use DARTS in Katib [here](https://github.com/kubeflow/katib/tree/master/pkg/suggestion/v1beta1/nas/darts).\n",
    "\n",
    "The notebook shows how to create, get, check status and delete an Experiment."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Install required package"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Collecting kubeflow-katib==0.10.1\n",
      "  Downloading kubeflow_katib-0.10.1-py3-none-any.whl (113 kB)\n",
      "\u001b[K     |████████████████████████████████| 113 kB 24.5 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: python-dateutil>=2.5.3 in /usr/local/lib/python3.6/dist-packages (from kubeflow-katib==0.10.1) (2.8.1)\n",
      "Requirement already satisfied: kubernetes==10.0.1 in /usr/local/lib/python3.6/dist-packages (from kubeflow-katib==0.10.1) (10.0.1)\n",
      "Requirement already satisfied: certifi>=14.05.14 in /usr/local/lib/python3.6/dist-packages (from kubeflow-katib==0.10.1) (2019.11.28)\n",
      "Requirement already satisfied: setuptools>=21.0.0 in /usr/local/lib/python3.6/dist-packages (from kubeflow-katib==0.10.1) (45.1.0)\n",
      "Requirement already satisfied: six>=1.10 in /usr/lib/python3/dist-packages (from kubeflow-katib==0.10.1) (1.11.0)\n",
      "Requirement already satisfied: urllib3>=1.15.1 in /usr/local/lib/python3.6/dist-packages (from kubeflow-katib==0.10.1) (1.25.8)\n",
      "Collecting table-logger>=0.3.5\n",
      "  Downloading table_logger-0.3.6-py3-none-any.whl (14 kB)\n",
      "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /usr/local/lib/python3.6/dist-packages (from kubernetes==10.0.1->kubeflow-katib==0.10.1) (0.57.0)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from kubernetes==10.0.1->kubeflow-katib==0.10.1) (2.22.0)\n",
      "Requirement already satisfied: requests-oauthlib in /usr/local/lib/python3.6/dist-packages (from kubernetes==10.0.1->kubeflow-katib==0.10.1) (1.3.0)\n",
      "Requirement already satisfied: google-auth>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from kubernetes==10.0.1->kubeflow-katib==0.10.1) (1.11.0)\n",
      "Requirement already satisfied: pyyaml>=3.12 in /usr/local/lib/python3.6/dist-packages (from kubernetes==10.0.1->kubeflow-katib==0.10.1) (5.3)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from table-logger>=0.3.5->kubeflow-katib==0.10.1) (1.18.1)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->kubernetes==10.0.1->kubeflow-katib==0.10.1) (3.0.4)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /usr/lib/python3/dist-packages (from requests->kubernetes==10.0.1->kubeflow-katib==0.10.1) (2.6)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from requests-oauthlib->kubernetes==10.0.1->kubeflow-katib==0.10.1) (3.1.0)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.0.1->kubernetes==10.0.1->kubeflow-katib==0.10.1) (4.0.0)\n",
      "Requirement already satisfied: rsa<4.1,>=3.1.4 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.0.1->kubernetes==10.0.1->kubeflow-katib==0.10.1) (4.0)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.6/dist-packages (from google-auth>=1.0.1->kubernetes==10.0.1->kubeflow-katib==0.10.1) (0.2.8)\n",
      "Requirement already satisfied: pyasn1>=0.1.3 in /usr/local/lib/python3.6/dist-packages (from rsa<4.1,>=3.1.4->google-auth>=1.0.1->kubernetes==10.0.1->kubeflow-katib==0.10.1) (0.4.8)\n",
      "Installing collected packages: table-logger, kubeflow-katib\n",
      "Successfully installed kubeflow-katib-0.10.1 table-logger-0.3.6\n",
      "\u001b[33mWARNING: You are using pip version 20.0.2; however, version 20.3 is available.\n",
      "You should consider upgrading via the '/usr/bin/python3 -m pip install --upgrade pip' command.\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install kubeflow-katib==0.10.1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Restart the Notebook kernel to use SDK package"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<script>Jupyter.notebook.kernel.restart()</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import display_html\n",
    "display_html(\"<script>Jupyter.notebook.kernel.restart()</script>\",raw=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import required packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from kubeflow.katib import KatibClient\n",
    "from kubernetes.client import V1ObjectMeta\n",
    "from kubeflow.katib import V1beta1Experiment\n",
    "from kubeflow.katib import V1beta1AlgorithmSpec\n",
    "from kubeflow.katib import V1beta1AlgorithmSetting\n",
    "from kubeflow.katib import V1beta1ObjectiveSpec\n",
    "from kubeflow.katib import V1beta1MetricsCollectorSpec\n",
    "from kubeflow.katib import V1beta1CollectorSpec\n",
    "from kubeflow.katib import V1beta1SourceSpec\n",
    "from kubeflow.katib import V1beta1FilterSpec\n",
    "from kubeflow.katib import V1beta1FeasibleSpace\n",
    "from kubeflow.katib import V1beta1ExperimentSpec\n",
    "from kubeflow.katib import V1beta1NasConfig\n",
    "from kubeflow.katib import V1beta1GraphConfig\n",
    "from kubeflow.katib import V1beta1Operation\n",
    "from kubeflow.katib import V1beta1ParameterSpec\n",
    "from kubeflow.katib import V1beta1TrialTemplate\n",
    "from kubeflow.katib import V1beta1TrialParameterSpec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define your Experiment\n",
    "\n",
    "You have to create your Experiment object before deploying it. This Experiment is similar to [this](https://github.com/kubeflow/katib/blob/master/examples/v1beta1/nas/darts-example-gpu.yaml) example.\n",
    "\n",
    "You can read more about DARTS algorithm settings [here](https://www.kubeflow.org/docs/components/katib/experiment/#differentiable-architecture-search-darts)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Experiment name and namespace.\n",
    "namespace = \"anonymous\"\n",
    "experiment_name = \"darts-example\"\n",
    "\n",
    "metadata = V1ObjectMeta(\n",
    "    name=experiment_name,\n",
    "    namespace=namespace\n",
    ")\n",
    "\n",
    "\n",
    "# Algorithm specification.\n",
    "algorithm_spec=V1beta1AlgorithmSpec(\n",
    "    algorithm_name=\"darts\",\n",
    "    algorithm_settings=[\n",
    "        V1beta1AlgorithmSetting(\n",
    "            name=\"num_epochs\",\n",
    "            value=\"2\"\n",
    "        ),\n",
    "        V1beta1AlgorithmSetting(\n",
    "            name=\"stem_multiplier\",\n",
    "            value=\"1\"\n",
    "        ),\n",
    "        V1beta1AlgorithmSetting(\n",
    "            name=\"init_channels\",\n",
    "            value=\"4\"\n",
    "        ),\n",
    "        V1beta1AlgorithmSetting(\n",
    "            name=\"num_nodes\",\n",
    "            value=\"3\"\n",
    "        ),\n",
    "        \n",
    "    ]\n",
    ")\n",
    "\n",
    "# Objective specification. For DARTS Goal is omitted.\n",
    "objective_spec=V1beta1ObjectiveSpec(\n",
    "    type=\"maximize\",\n",
    "    objective_metric_name=\"Best-Genotype\",\n",
    ")\n",
    "\n",
    "# Metrics collector specification.\n",
    "# We should specify metrics format to get Genotype from training container.\n",
    "metrics_collector_spec=V1beta1MetricsCollectorSpec(\n",
    "    collector=V1beta1CollectorSpec(\n",
    "        kind=\"StdOut\"\n",
    "    ),\n",
    "    source=V1beta1SourceSpec(\n",
    "        filter=V1beta1FilterSpec(\n",
    "            metrics_format=[\n",
    "                \"([\\\\w-]+)=(Genotype.*)\"\n",
    "            ]\n",
    "        )\n",
    "    )\n",
    ")\n",
    "\n",
    "# Configuration for the Neural Network (NN).\n",
    "# This NN contains 2 number of layers and 5 various operations with different parameters.\n",
    "nas_config=V1beta1NasConfig(\n",
    "    graph_config=V1beta1GraphConfig(\n",
    "        num_layers=2\n",
    "    ),\n",
    "    operations=[\n",
    "        V1beta1Operation(\n",
    "            operation_type=\"separable_convolution\",\n",
    "            parameters=[\n",
    "                V1beta1ParameterSpec(\n",
    "                    name=\"filter_size\",\n",
    "                    parameter_type=\"categorical\",\n",
    "                    feasible_space=V1beta1FeasibleSpace(\n",
    "                        list=[\"3\"]\n",
    "                    ),\n",
    "                )\n",
    "            ]\n",
    "        ),\n",
    "        V1beta1Operation(\n",
    "            operation_type=\"dilated_convolution\",\n",
    "            parameters=[\n",
    "                V1beta1ParameterSpec(\n",
    "                    name=\"filter_size\",\n",
    "                    parameter_type=\"categorical\",\n",
    "                    feasible_space=V1beta1FeasibleSpace(\n",
    "                        list=[\"3\", \"5\"]\n",
    "                    ),\n",
    "                )\n",
    "            ]\n",
    "        ),\n",
    "        V1beta1Operation(\n",
    "            operation_type=\"avg_pooling\",\n",
    "            parameters=[\n",
    "                V1beta1ParameterSpec(\n",
    "                    name=\"filter_size\",\n",
    "                    parameter_type=\"categorical\",\n",
    "                    feasible_space=V1beta1FeasibleSpace(\n",
    "                        list=[\"3\"]\n",
    "                    ),\n",
    "                )\n",
    "            ]\n",
    "        ),\n",
    "        V1beta1Operation(\n",
    "            operation_type=\"max_pooling\",\n",
    "            parameters=[\n",
    "                V1beta1ParameterSpec(\n",
    "                    name=\"filter_size\",\n",
    "                    parameter_type=\"categorical\",\n",
    "                    feasible_space=V1beta1FeasibleSpace(\n",
    "                        list=[\"3\"]\n",
    "                    ),\n",
    "                )\n",
    "            ]\n",
    "        ),\n",
    "        V1beta1Operation(\n",
    "            operation_type=\"skip_connection\",\n",
    "        ),\n",
    "    ]\n",
    ")\n",
    "\n",
    "\n",
    "# JSON template specification for the Trial's Worker Kubernetes Job.\n",
    "trial_spec={\n",
    "    \"apiVersion\": \"batch/v1\",\n",
    "    \"kind\": \"Job\",\n",
    "    \"spec\": {\n",
    "        \"template\": {\n",
    "            \"metadata\": {\n",
    "                \"annotations\": {\n",
    "                    \"sidecar.istio.io/inject\": \"false\"\n",
    "                }\n",
    "            },\n",
    "            \"spec\": {\n",
    "                \"containers\": [\n",
    "                    {\n",
    "                        \"name\": \"training-container\",\n",
    "                        \"image\": \"docker.io/kubeflowkatib/darts-cnn-cifar10:v1beta1-45c5727\",\n",
    "                        \"command\": [\n",
    "                            'python3',\n",
    "                            'run_trial.py',\n",
    "                            '--algorithm-settings=\"${trialParameters.algorithmSettings}\"',\n",
    "                            '--search-space=\"${trialParameters.searchSpace}\"',\n",
    "                            '--num-layers=\"${trialParameters.numberLayers}\"'\n",
    "                        ],\n",
    "                        # Training container requires 1 GPU.\n",
    "                        \"resources\": {\n",
    "                            \"limits\": {\n",
    "                                \"nvidia.com/gpu\": 1\n",
    "                            }\n",
    "                        }\n",
    "                    }\n",
    "                ],\n",
    "                \"restartPolicy\": \"Never\"\n",
    "            }\n",
    "        }\n",
    "    }\n",
    "}\n",
    "\n",
    "# Template with Trial parameters and Trial spec.\n",
    "# Set retain to True to save trial resources after completion.\n",
    "trial_template=V1beta1TrialTemplate(\n",
    "    retain=True,\n",
    "    primary_container_name=\"training-container\",\n",
    "    trial_parameters=[\n",
    "        V1beta1TrialParameterSpec(\n",
    "            name=\"algorithmSettings\",\n",
    "            description=\" Algorithm settings of DARTS Experiment\",\n",
    "            reference=\"algorithm-settings\"\n",
    "        ),\n",
    "        V1beta1TrialParameterSpec(\n",
    "            name=\"searchSpace\",\n",
    "            description=\"Search Space of DARTS Experiment\",\n",
    "            reference=\"search-space\"\n",
    "        ),\n",
    "        V1beta1TrialParameterSpec(\n",
    "            name=\"numberLayers\",\n",
    "            description=\"Number of Neural Network layers\",\n",
    "            reference=\"num-layers\"\n",
    "        ),\n",
    "    ],\n",
    "    trial_spec=trial_spec\n",
    ")\n",
    "\n",
    "\n",
    "# Experiment object.\n",
    "experiment = V1beta1Experiment(\n",
    "    api_version=\"kubeflow.org/v1beta1\",\n",
    "    kind=\"Experiment\",\n",
    "    metadata=metadata,\n",
    "    spec=V1beta1ExperimentSpec(\n",
    "        max_trial_count=1,\n",
    "        parallel_trial_count=1,\n",
    "        max_failed_trial_count=1,\n",
    "        algorithm=algorithm_spec,\n",
    "        objective=objective_spec,\n",
    "        metrics_collector_spec=metrics_collector_spec,\n",
    "        nas_config=nas_config,\n",
    "        trial_template=trial_template,\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can print the Experiment's info to verify it before submission."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'name': 'training-container', 'image': 'docker.io/kubeflowkatib/darts-cnn-cifar10:v1beta1-45c5727', 'command': ['python3', 'run_trial.py', '--algorithm-settings=\"${trialParameters.algorithmSettings}\"', '--search-space=\"${trialParameters.searchSpace}\"', '--num-layers=\"${trialParameters.numberLayers}\"'], 'resources': {'limits': {'nvidia.com/gpu': 1}}}\n"
     ]
    }
   ],
   "source": [
    "# Print the Trial template container info.\n",
    "print(experiment.spec.trial_template.trial_spec[\"spec\"][\"template\"][\"spec\"][\"containers\"][0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create your Experiment\n",
    "\n",
    "You have to create Katib client to use the SDK\n",
    "\n",
    "TODO (andreyvelich): Current Experiment link for NAS is incorrect."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "Katib Experiment darts-example link <a href=\"/_/katib/#/katib/hp_monitor/anonymous/darts-example\" target=\"_blank\">here</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'apiVersion': 'kubeflow.org/v1beta1',\n",
       " 'kind': 'Experiment',\n",
       " 'metadata': {'creationTimestamp': '2020-11-30T19:32:50Z',\n",
       "  'generation': 1,\n",
       "  'name': 'darts-example',\n",
       "  'namespace': 'anonymous',\n",
       "  'resourceVersion': '170793314',\n",
       "  'selfLink': '/apis/kubeflow.org/v1beta1/namespaces/anonymous/experiments/darts-example',\n",
       "  'uid': '00cd64b7-9388-4faf-93e2-5667989f723d'},\n",
       " 'spec': {'algorithm': {'algorithmName': 'darts',\n",
       "   'algorithmSettings': [{'name': 'num_epochs', 'value': '2'},\n",
       "    {'name': 'stem_multiplier', 'value': '1'},\n",
       "    {'name': 'init_channels', 'value': '4'},\n",
       "    {'name': 'num_nodes', 'value': '3'}]},\n",
       "  'maxFailedTrialCount': 1,\n",
       "  'maxTrialCount': 1,\n",
       "  'metricsCollectorSpec': {'collector': {'kind': 'StdOut'},\n",
       "   'source': {'filter': {'metricsFormat': ['([\\\\w-]+)=(Genotype.*)']}}},\n",
       "  'nasConfig': {'graphConfig': {'numLayers': 2},\n",
       "   'operations': [{'operationType': 'separable_convolution',\n",
       "     'parameters': [{'feasibleSpace': {'list': ['3']},\n",
       "       'name': 'filter_size',\n",
       "       'parameterType': 'categorical'}]},\n",
       "    {'operationType': 'dilated_convolution',\n",
       "     'parameters': [{'feasibleSpace': {'list': ['3', '5']},\n",
       "       'name': 'filter_size',\n",
       "       'parameterType': 'categorical'}]},\n",
       "    {'operationType': 'avg_pooling',\n",
       "     'parameters': [{'feasibleSpace': {'list': ['3']},\n",
       "       'name': 'filter_size',\n",
       "       'parameterType': 'categorical'}]},\n",
       "    {'operationType': 'max_pooling',\n",
       "     'parameters': [{'feasibleSpace': {'list': ['3']},\n",
       "       'name': 'filter_size',\n",
       "       'parameterType': 'categorical'}]},\n",
       "    {'operationType': 'skip_connection'}]},\n",
       "  'objective': {'metricStrategies': [{'name': 'Best-Genotype',\n",
       "     'value': 'max'}],\n",
       "   'objectiveMetricName': 'Best-Genotype',\n",
       "   'type': 'maximize'},\n",
       "  'parallelTrialCount': 1,\n",
       "  'resumePolicy': 'LongRunning',\n",
       "  'trialTemplate': {'failureCondition': 'status.conditions.#(type==\"Failed\")#|#(status==\"True\")#',\n",
       "   'primaryContainerName': 'training-container',\n",
       "   'retain': True,\n",
       "   'successCondition': 'status.conditions.#(type==\"Complete\")#|#(status==\"True\")#',\n",
       "   'trialParameters': [{'description': ' Algorithm settings of DARTS Experiment',\n",
       "     'name': 'algorithmSettings',\n",
       "     'reference': 'algorithm-settings'},\n",
       "    {'description': 'Search Space of DARTS Experiment',\n",
       "     'name': 'searchSpace',\n",
       "     'reference': 'search-space'},\n",
       "    {'description': 'Number of Neural Network layers',\n",
       "     'name': 'numberLayers',\n",
       "     'reference': 'num-layers'}],\n",
       "   'trialSpec': {'apiVersion': 'batch/v1',\n",
       "    'kind': 'Job',\n",
       "    'spec': {'template': {'metadata': {'annotations': {'sidecar.istio.io/inject': 'false'}},\n",
       "      'spec': {'containers': [{'command': ['python3',\n",
       "          'run_trial.py',\n",
       "          '--algorithm-settings=\"${trialParameters.algorithmSettings}\"',\n",
       "          '--search-space=\"${trialParameters.searchSpace}\"',\n",
       "          '--num-layers=\"${trialParameters.numberLayers}\"'],\n",
       "         'image': 'docker.io/kubeflowkatib/darts-cnn-cifar10:v1beta1-45c5727',\n",
       "         'name': 'training-container',\n",
       "         'resources': {'limits': {'nvidia.com/gpu': 1}}}],\n",
       "       'restartPolicy': 'Never'}}}}}}}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create client.\n",
    "kclient = KatibClient()\n",
    "\n",
    "# Create your Experiment.\n",
    "kclient.create_experiment(experiment,namespace=namespace)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get your Experiment\n",
    "\n",
    "You can get your Experiment by name and receive required data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'apiVersion': 'kubeflow.org/v1beta1', 'kind': 'Experiment', 'metadata': {'creationTimestamp': '2020-11-30T19:32:50Z', 'finalizers': ['update-prometheus-metrics'], 'generation': 1, 'name': 'darts-example', 'namespace': 'anonymous', 'resourceVersion': '170793467', 'selfLink': '/apis/kubeflow.org/v1beta1/namespaces/anonymous/experiments/darts-example', 'uid': '00cd64b7-9388-4faf-93e2-5667989f723d'}, 'spec': {'algorithm': {'algorithmName': 'darts', 'algorithmSettings': [{'name': 'num_epochs', 'value': '2'}, {'name': 'stem_multiplier', 'value': '1'}, {'name': 'init_channels', 'value': '4'}, {'name': 'num_nodes', 'value': '3'}]}, 'maxFailedTrialCount': 1, 'maxTrialCount': 1, 'metricsCollectorSpec': {'collector': {'kind': 'StdOut'}, 'source': {'filter': {'metricsFormat': ['([\\\\w-]+)=(Genotype.*)']}}}, 'nasConfig': {'graphConfig': {'numLayers': 2}, 'operations': [{'operationType': 'separable_convolution', 'parameters': [{'feasibleSpace': {'list': ['3']}, 'name': 'filter_size', 'parameterType': 'categorical'}]}, {'operationType': 'dilated_convolution', 'parameters': [{'feasibleSpace': {'list': ['3', '5']}, 'name': 'filter_size', 'parameterType': 'categorical'}]}, {'operationType': 'avg_pooling', 'parameters': [{'feasibleSpace': {'list': ['3']}, 'name': 'filter_size', 'parameterType': 'categorical'}]}, {'operationType': 'max_pooling', 'parameters': [{'feasibleSpace': {'list': ['3']}, 'name': 'filter_size', 'parameterType': 'categorical'}]}, {'operationType': 'skip_connection'}]}, 'objective': {'metricStrategies': [{'name': 'Best-Genotype', 'value': 'max'}], 'objectiveMetricName': 'Best-Genotype', 'type': 'maximize'}, 'parallelTrialCount': 1, 'resumePolicy': 'LongRunning', 'trialTemplate': {'failureCondition': 'status.conditions.#(type==\"Failed\")#|#(status==\"True\")#', 'primaryContainerName': 'training-container', 'retain': True, 'successCondition': 'status.conditions.#(type==\"Complete\")#|#(status==\"True\")#', 'trialParameters': [{'description': ' Algorithm settings of DARTS Experiment', 'name': 'algorithmSettings', 'reference': 'algorithm-settings'}, {'description': 'Search Space of DARTS Experiment', 'name': 'searchSpace', 'reference': 'search-space'}, {'description': 'Number of Neural Network layers', 'name': 'numberLayers', 'reference': 'num-layers'}], 'trialSpec': {'apiVersion': 'batch/v1', 'kind': 'Job', 'spec': {'template': {'metadata': {'annotations': {'sidecar.istio.io/inject': 'false'}}, 'spec': {'containers': [{'command': ['python3', 'run_trial.py', '--algorithm-settings=\"${trialParameters.algorithmSettings}\"', '--search-space=\"${trialParameters.searchSpace}\"', '--num-layers=\"${trialParameters.numberLayers}\"'], 'image': 'docker.io/kubeflowkatib/darts-cnn-cifar10:v1beta1-45c5727', 'name': 'training-container', 'resources': {'limits': {'nvidia.com/gpu': 1}}}], 'restartPolicy': 'Never'}}}}}}, 'status': {'conditions': [{'lastTransitionTime': '2020-11-30T19:32:50Z', 'lastUpdateTime': '2020-11-30T19:32:50Z', 'message': 'Experiment is created', 'reason': 'ExperimentCreated', 'status': 'True', 'type': 'Created'}, {'lastTransitionTime': '2020-11-30T19:33:07Z', 'lastUpdateTime': '2020-11-30T19:33:07Z', 'message': 'Experiment is running', 'reason': 'ExperimentRunning', 'status': 'True', 'type': 'Running'}], 'currentOptimalTrial': {'bestTrialName': '', 'observation': {'metrics': None}, 'parameterAssignments': None}, 'runningTrialList': ['darts-example-t8qb74sn'], 'startTime': '2020-11-30T19:32:50Z', 'trials': 1, 'trialsRunning': 1}}\n",
      "-----------------\n",
      "\n",
      "{'lastTransitionTime': '2020-11-30T19:33:07Z', 'lastUpdateTime': '2020-11-30T19:33:07Z', 'message': 'Experiment is running', 'reason': 'ExperimentRunning', 'status': 'True', 'type': 'Running'}\n"
     ]
    }
   ],
   "source": [
    "exp = kclient.get_experiment(name=experiment_name, namespace=namespace)\n",
    "print(exp)\n",
    "print(\"-----------------\\n\")\n",
    "\n",
    "# Get the latest status.\n",
    "print(exp[\"status\"][\"conditions\"][-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get the current Experiment status\n",
    "\n",
    "You can check the current Experiment status."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Succeeded'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kclient.get_experiment_status(name=experiment_name, namespace=namespace)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can check if your Experiment is succeeded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kclient.is_experiment_succeeded(name=experiment_name, namespace=namespace)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Get the best Genotype\n",
    "\n",
    "Best Genotype is located in the optimal Trial currently. The latest Genotype is the best.\n",
    "\n",
    "Check your Trial logs to get more information about the training process."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Genotype(normal=[[('separable_convolution_3x3',0),('dilated_convolution_3x3',1)],[('dilated_convolution_3x3',2),('dilated_convolution_5x5',1)],[('dilated_convolution_5x5',2),('separable_convolution_3x3',3)]],normal_concat=range(2,5),reduce=[[('separable_convolution_3x3',1),('separable_convolution_3x3',0)],[('max_pooling_3x3',2),('max_pooling_3x3',1)],[('max_pooling_3x3',2),('max_pooling_3x3',3)]],reduce_concat=range(2,5))\n"
     ]
    }
   ],
   "source": [
    "opt_trial = kclient.get_optimal_hyperparameters(name=experiment_name, namespace=namespace)\n",
    "\n",
    "best_genotype = opt_trial[\"currentOptimalTrial\"][\"observation\"][\"metrics\"][0][\"latest\"]\n",
    "print(best_genotype)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Delete your Experiments\n",
    "\n",
    "You can delete your Experiments."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'apiVersion': 'kubeflow.org/v1beta1',\n",
       " 'kind': 'Experiment',\n",
       " 'metadata': {'creationTimestamp': '2020-11-30T19:32:50Z',\n",
       "  'deletionGracePeriodSeconds': 0,\n",
       "  'deletionTimestamp': '2020-11-30T19:36:58Z',\n",
       "  'finalizers': ['update-prometheus-metrics'],\n",
       "  'generation': 2,\n",
       "  'name': 'darts-example',\n",
       "  'namespace': 'anonymous',\n",
       "  'resourceVersion': '170795247',\n",
       "  'selfLink': '/apis/kubeflow.org/v1beta1/namespaces/anonymous/experiments/darts-example',\n",
       "  'uid': '00cd64b7-9388-4faf-93e2-5667989f723d'},\n",
       " 'spec': {'algorithm': {'algorithmName': 'darts',\n",
       "   'algorithmSettings': [{'name': 'num_epochs', 'value': '2'},\n",
       "    {'name': 'stem_multiplier', 'value': '1'},\n",
       "    {'name': 'init_channels', 'value': '4'},\n",
       "    {'name': 'num_nodes', 'value': '3'}]},\n",
       "  'maxFailedTrialCount': 1,\n",
       "  'maxTrialCount': 1,\n",
       "  'metricsCollectorSpec': {'collector': {'kind': 'StdOut'},\n",
       "   'source': {'filter': {'metricsFormat': ['([\\\\w-]+)=(Genotype.*)']}}},\n",
       "  'nasConfig': {'graphConfig': {'numLayers': 2},\n",
       "   'operations': [{'operationType': 'separable_convolution',\n",
       "     'parameters': [{'feasibleSpace': {'list': ['3']},\n",
       "       'name': 'filter_size',\n",
       "       'parameterType': 'categorical'}]},\n",
       "    {'operationType': 'dilated_convolution',\n",
       "     'parameters': [{'feasibleSpace': {'list': ['3', '5']},\n",
       "       'name': 'filter_size',\n",
       "       'parameterType': 'categorical'}]},\n",
       "    {'operationType': 'avg_pooling',\n",
       "     'parameters': [{'feasibleSpace': {'list': ['3']},\n",
       "       'name': 'filter_size',\n",
       "       'parameterType': 'categorical'}]},\n",
       "    {'operationType': 'max_pooling',\n",
       "     'parameters': [{'feasibleSpace': {'list': ['3']},\n",
       "       'name': 'filter_size',\n",
       "       'parameterType': 'categorical'}]},\n",
       "    {'operationType': 'skip_connection'}]},\n",
       "  'objective': {'metricStrategies': [{'name': 'Best-Genotype',\n",
       "     'value': 'max'}],\n",
       "   'objectiveMetricName': 'Best-Genotype',\n",
       "   'type': 'maximize'},\n",
       "  'parallelTrialCount': 1,\n",
       "  'resumePolicy': 'LongRunning',\n",
       "  'trialTemplate': {'failureCondition': 'status.conditions.#(type==\"Failed\")#|#(status==\"True\")#',\n",
       "   'primaryContainerName': 'training-container',\n",
       "   'retain': True,\n",
       "   'successCondition': 'status.conditions.#(type==\"Complete\")#|#(status==\"True\")#',\n",
       "   'trialParameters': [{'description': ' Algorithm settings of DARTS Experiment',\n",
       "     'name': 'algorithmSettings',\n",
       "     'reference': 'algorithm-settings'},\n",
       "    {'description': 'Search Space of DARTS Experiment',\n",
       "     'name': 'searchSpace',\n",
       "     'reference': 'search-space'},\n",
       "    {'description': 'Number of Neural Network layers',\n",
       "     'name': 'numberLayers',\n",
       "     'reference': 'num-layers'}],\n",
       "   'trialSpec': {'apiVersion': 'batch/v1',\n",
       "    'kind': 'Job',\n",
       "    'spec': {'template': {'metadata': {'annotations': {'sidecar.istio.io/inject': 'false'}},\n",
       "      'spec': {'containers': [{'command': ['python3',\n",
       "          'run_trial.py',\n",
       "          '--algorithm-settings=\"${trialParameters.algorithmSettings}\"',\n",
       "          '--search-space=\"${trialParameters.searchSpace}\"',\n",
       "          '--num-layers=\"${trialParameters.numberLayers}\"'],\n",
       "         'image': 'docker.io/kubeflowkatib/darts-cnn-cifar10:v1beta1-45c5727',\n",
       "         'name': 'training-container',\n",
       "         'resources': {'limits': {'nvidia.com/gpu': 1}}}],\n",
       "       'restartPolicy': 'Never'}}}}}},\n",
       " 'status': {'completionTime': '2020-11-30T19:36:29Z',\n",
       "  'conditions': [{'lastTransitionTime': '2020-11-30T19:32:50Z',\n",
       "    'lastUpdateTime': '2020-11-30T19:32:50Z',\n",
       "    'message': 'Experiment is created',\n",
       "    'reason': 'ExperimentCreated',\n",
       "    'status': 'True',\n",
       "    'type': 'Created'},\n",
       "   {'lastTransitionTime': '2020-11-30T19:36:29Z',\n",
       "    'lastUpdateTime': '2020-11-30T19:36:29Z',\n",
       "    'message': 'Experiment is running',\n",
       "    'reason': 'ExperimentRunning',\n",
       "    'status': 'False',\n",
       "    'type': 'Running'},\n",
       "   {'lastTransitionTime': '2020-11-30T19:36:29Z',\n",
       "    'lastUpdateTime': '2020-11-30T19:36:29Z',\n",
       "    'message': 'Experiment has succeeded because max trial count has reached',\n",
       "    'reason': 'ExperimentMaxTrialsReached',\n",
       "    'status': 'True',\n",
       "    'type': 'Succeeded'}],\n",
       "  'currentOptimalTrial': {'bestTrialName': 'darts-example-t8qb74sn',\n",
       "   'observation': {'metrics': [{'latest': \"Genotype(normal=[[('separable_convolution_3x3',0),('dilated_convolution_3x3',1)],[('dilated_convolution_3x3',2),('dilated_convolution_5x5',1)],[('dilated_convolution_5x5',2),('separable_convolution_3x3',3)]],normal_concat=range(2,5),reduce=[[('separable_convolution_3x3',1),('separable_convolution_3x3',0)],[('max_pooling_3x3',2),('max_pooling_3x3',1)],[('max_pooling_3x3',2),('max_pooling_3x3',3)]],reduce_concat=range(2,5))\",\n",
       "      'max': 'unavailable',\n",
       "      'min': 'unavailable',\n",
       "      'name': 'Best-Genotype'}]},\n",
       "   'parameterAssignments': [{'name': 'algorithm-settings',\n",
       "     'value': \"{'num_epochs': '2', 'w_lr': 0.025, 'w_lr_min': 0.001, 'w_momentum': 0.9, 'w_weight_decay': 0.0003, 'w_grad_clip': 5.0, 'alpha_lr': 0.0003, 'alpha_weight_decay': 0.001, 'batch_size': 128, 'num_workers': 4, 'init_channels': '4', 'print_step': 50, 'num_nodes': '3', 'stem_multiplier': '1'}\"},\n",
       "    {'name': 'search-space',\n",
       "     'value': \"['separable_convolution_3x3', 'dilated_convolution_3x3', 'dilated_convolution_5x5', 'avg_pooling_3x3', 'max_pooling_3x3', 'skip_connection']\"},\n",
       "    {'name': 'num-layers', 'value': '2'}]},\n",
       "  'startTime': '2020-11-30T19:32:50Z',\n",
       "  'succeededTrialList': ['darts-example-t8qb74sn'],\n",
       "  'trials': 1,\n",
       "  'trialsSucceeded': 1}}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kclient.delete_experiment(name=experiment_name, namespace=namespace)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
